/*
 * CompanyDB.cpp
 *
 *  Created on: Mar 25, 2014
 *      Author: Alexander
 */

#include "CompanyDB.h"
#include <algorithm>
#include <map>

#include <Poco/StringTokenizer.h>
#include <Poco/String.h>
#include <Poco/NumberParser.h>
#include "Poco/ThreadPool.h"
#include "Poco/Runnable.h"
#include "Poco/RWLock.h"
#include "Poco/Util/XMLConfiguration.h"
#include "Poco/AutoPtr.h"

#include "CompanyData.h"

const std::vector<std::string> CompanyDB::EXCHANGES = {"NYSE", "NASDAQ", "AMEX"};
const std::string CompanyDB::EXCHANGE_LIST_URL_1 = "www.nasdaq.com/screening/companies-by-industry.aspx?exchange=";
const std::string CompanyDB::EXCHANGE_LIST_URL_2 = "&render=download";
/*
const std::string CompanyDB::FIN_STATEMENT_URL_1 = "www.nasdaq.com/symbol/";
const std::string CompanyDB::FIN_STATEMENT_URL_2 = "/financials?query=";

const std::string CompanyDB::INCOME_STATEMENT_QUERY_URL = "income-statement";
const std::string CompanyDB::BALANCE_SHEET_QUERY_URL = "balance-sheet";
const std::string CompanyDB::CASH_FLOW_QUERY_URL = "cash-flow";
const std::string CompanyDB::FIN_RATIO_QUERY_URL = "ratios";

const std::string CompanyDB::KEY_STATS_URL_1 = "finance.yahoo.com/q/ks?s=";
const std::string CompanyDB::KEY_STATS_URL_2 = "+Key+Statistics";
*/

//const std::string FIN_STATEMENT_URL_1 = "http://www.nasdaq.com/symbol/";
const std::string FIN_STATEMENT_URL_1 = "http://finance.yahoo.com/q/is?s=";
//const std::string FIN_STATEMENT_URL_2 = "/financials?query=";
const std::string FIN_STATEMENT_URL_2 = "+Income+Statement&annual";

const std::string INCOME_STATEMENT_QUERY_URL = "income-statement";
const std::string BALANCE_SHEET_QUERY_URL = "balance-sheet";
const std::string CASH_FLOW_QUERY_URL = "cash-flow";
const std::string FIN_RATIO_QUERY_URL = "ratios";

const std::string KEY_STATS_URL_1 = "finance.yahoo.com/q/ks?s=";
const std::string KEY_STATS_URL_2 = "+Key+Statistics";

const std::string GOOG_FINANCIAL_1 = "www.google.com/finance?q=";
const std::string GOOG_FINANCIAL_2 = "%3A";
const std::string GOOG_FINANCIAL_3 = "&fstype=ii";


class ExchangeCompaniesThread: public Poco::Runnable
{
public:
	ExchangeCompaniesThread (std::map<std::string, CompanyData> * db, const std::string& exchangeName,
			Poco::RWLock * lock):
		companies_ (db), exchange (exchangeName), lock_ (lock)
	{ }

	virtual void run()
	{
		std::map<std::string, CompanyData> tmp = CompanyDB::populateCompanyList(exchange);
		lock_->writeLock();
		companies_->insert(tmp.begin(), tmp.end());
		lock_->unlock();
	}
private:
	std::map<std::string, CompanyData> * companies_;
	std::string exchange;
	Poco::RWLock * lock_;
};

enum FinDataCmd
{
	INCOME_STATEMENT = 0,
	BALANCE_SHEET = 1,
	CASH_FLOW = 2,
	FIN_RATIOS = 3,
	KEY_STATS = 4,
};

std::vector<std::string> retrieveYearsAndMultiplier(const std::vector<std::string>& fetchedData, int counter,
		std::vector<std::string>& years, int& multiplier)
{
	const std::string year_str = "12 months ending";
	const std::string multi_str = "except for per share items";
	const std::string million_str = "Millions";
	const std::string thousand_str = "Thousands";


	const std::string marker = "<thead><tr><th class=\"lm lft nwp\">";

	int count = 0;

	//std::vector<std::string>::const_iterator start;
	auto start = fetchedData.begin();
	auto end = fetchedData.end();
	//std::vector<std::string>::const_iterator end;
	for (auto it = fetchedData.begin(); it != fetchedData.end(); ++it)
	{
		if (Poco::icompare(*it, marker) == 0)
		{
			if (count > counter * 2)
			{
				if (count > counter * 2 + 1)
				{
					end = it;
					break;
				}
				start = it;
			}
			count++;
		}
	}

	std::vector<std::string> extracted(start, end);

	auto it = extracted.begin();
	auto end_it = extracted.end();

//	for (auto a : extracted)
//	{
//		std::cout << a << std::endl;
//	}

	for (; it != end_it; ++it)
	{
		if (Poco::isubstr(*it, multi_str) != std::string::npos)
		{
			if (Poco::isubstr(*it, million_str) != std::string::npos)
			{
				multiplier = 1000000;
			}
			else if (Poco::isubstr(*it, thousand_str) != std::string::npos)
			{
				multiplier = 1000;
			}
			else
			{
				std::cout << "Error in multiplier" << std::endl;
				throw std::exception();
			}
		}
//	}
//
//	for (; it != end_it; ++it)
//	{
		if (Poco::isubstr(*it, year_str) != std::string::npos)
		{
			Poco::RegularExpression re("[12][0-9][0-9][0-9]-[01]?[0-9]-[0-3]?[0-9]");
			std::string str;
			int n = re.extract(*it, str);
			if (n == 1)
			{
				years.push_back(str);
				counter++;
			}
			else
			{
				std::cout << "Error while parsing year: " << *it << std::endl;
				throw std::exception();
			}
		}
	}
	return extracted;
}

class FinDataThread: public Poco::Runnable
{
public:
	FinDataThread (const std::string& ticker, FinDataCmd cmd, const std::vector<std::string>& fetchedData,
			const std::vector<std::string>& categories, const std::vector<std::string>& categoriesPS,
			std::map<std::string, std::map<std::string, float>>* financials):
				ticker_(ticker), cmd_(cmd), fetchedData_(fetchedData), categories_(categories),
				categoriesPS_(categoriesPS), financials_(financials)
	{ }

	virtual void run()
	{
		int multiplier;
		std::vector<std::string> years;
		std::vector<std::string> data;
		switch (cmd_)
		{
		case INCOME_STATEMENT:
		case BALANCE_SHEET:
		case CASH_FLOW:
			try
			{
				data = retrieveYearsAndMultiplier(fetchedData_, cmd_, years, multiplier);
				CompanyDB::retrieveFinancials(data, years, categories_, categoriesPS_, multiplier, financials_);
			}
			catch(std::exception& e)
			{
				std::cout << "Error for: " << ticker_ << "!!!" << std::endl;
			}
			break;

		case KEY_STATS:
//				{
//					fetchedData = d.fetchData(KEY_STATS_URL_1 + tickerInLower + KEY_STATS_URL_2);
//					KeyStatistics ks;
//					mutex_->lock();
//					cd = db_->at(ticker_);
//					cd.setKeyStatistics(ks.parseWebPageAndSet(fetchedData));
//					mutex_->unlock();
//					(*db_)[ticker_] = cd;
//					return;
//				}
			break;
		default:
			return;
		}
	}

private:
	std::string ticker_;
	FinDataCmd cmd_;
	std::vector<std::string> fetchedData_;
	std::vector<std::string> categories_;
	std::vector<std::string> categoriesPS_;
	std::map<std::string, std::map<std::string, float>>* financials_;
};


void CompanyDB::parseFinancialsWebPage(std::vector<std::string> fetchedData, IncomeStatement& is)
{
	std::string year_str = "12 months ending";
	std::string multi_str = "except for per share items";

	int ind = 0;
	int offs = 4;
	int counter = 0;
	int counter2 = 0;
	const int SIZE = IncomeStatement::CATEGORIES.size();
	std::vector<unsigned char> flags(SIZE, 0);
	std::vector<std::string> years;

	int multiplier = 1;

	auto it = fetchedData.begin();
	auto end_it = fetchedData.end();

	for (; it != end_it; ++it)
	{
		if (counter >= ind * offs && counter < (ind + 1) * offs)
		{
//			std::cout << *it << std::endl;
			if (Poco::isubstr(*it, year_str) != std::string::npos)
			{
//				std::cout << *it << std::endl;
				Poco::RegularExpression re("[12][0-9][0-9][0-9]-[01]?[0-9]-[0-3]?[0-9]");
				std::string str;
				int n = re.extract(*it, str);
				if (n == 1)
				{
					years.push_back(str);
					counter++;
				}
				else
				{
					std::cout << "Error while parsing year: " << *it << std::endl;
				}
			}
		}
		else
		{
			break;
		}
	}

	for (++it; it != end_it; ++it)
	{
		if (Poco::isubstr(*it, year_str) != std::string::npos)
		{
			break;
		}

		for (int i = 0; i < SIZE; ++i)
		{
			if (flags[i])
			{
				continue;
			}
			if (Poco::isubstr(*it, IncomeStatement::CATEGORIES[i]) != std::string::npos)
			{
//				std::cout << *it << std::endl;
				it += 2;
//				std::cout << *it << std::endl;
				Poco::RegularExpression re("[-]?[0-9][0-9,\.]*");
				std::string str;
				int n;
				for (int j = 0; j < years.size(); ++j)
				{
					n = re.extract(*it, str);
					if (n == 1)
					{
						if (str.find("-") != std::string::npos)
						{
							multiplier *= -1;
							Poco::translateInPlace(str, "-", "");
						}
						Poco::translateInPlace(str, ",", "");
						float value = multiplier * Poco::NumberParser::parseFloat(str);
						is.setValue(IncomeStatement::CATEGORIES[i], years[j], value);
						flags[i] = 1;
					}
					else
					{
						Poco::RegularExpression re(">-<");
						n = re.extract(*it, str);
						if (n == 1)
						{
							flags[i] = 1;
						}
						else
						{
							std::cout << "!!! Error: " << *it << std::endl;
						}
					}
					it++;
				}
			}
		}
	}
}

void CompanyDB::populateCompanyList()
{
	Downloader d("wwwgate0.mot.com", 1080);

	std::vector<ExchangeCompaniesThread> threads;

	Poco::RWLock lock;
	for (auto exchange = EXCHANGES.begin(); exchange != EXCHANGES.end(); exchange++)
	{
		threads.push_back(ExchangeCompaniesThread(&companies, *exchange, &lock));
	}

	Poco::ThreadPool pool;
	for (auto thread = threads.begin(); thread != threads.end(); thread++)
	{
		pool.start(*thread);
	}
	pool.joinAll();
}


//void CompanyDB::populateCompanyList(const std::string& exchange)
std::map<std::string, CompanyData> CompanyDB::populateCompanyList(const std::string& exchange)
{
	//Downloader d("wwwgate0.mot.com", 1080);
	//populateCompanyList(d, exchange);
	return populateCompanyList(Downloader("wwwgate0.mot.com", 1080), exchange);
}

//void CompanyDB::populateCompanyList(const Downloader& d, const std::string& exchange)
std::map<std::string, CompanyData> CompanyDB::populateCompanyList(const Downloader& d, const std::string& exchange)
{
	enum
	{
		SYMBOL_INDEX = 0,
		NAME_INDEX,
		LAST_SALE_INDEX,
		MARKET_CAP_INDEX,
		ADR_TSO_INDEX,
		IPO_YEAR_INDEX,
		SECTOR_INDEX,
		INDUSTRY_INDEX,
		SUMMARY_QUOTE_INDEX,
	};

	std::vector<std::string> fetchedData =
			d.fetchData(CompanyDB::EXCHANGE_LIST_URL_1 + exchange + CompanyDB::EXCHANGE_LIST_URL_2);
//	std::vector<std::string> fetchedData;
//	d.fetchData(CompanyDB::EXCHANGE_LIST_URL_1 + exchange + CompanyDB::EXCHANGE_LIST_URL_2, fetchedData);
	// TODO validate

	std::map<std::string, CompanyData> companiesOfExchange;

	for_each (fetchedData.begin() + 1, fetchedData.end(),
			[&](const std::string& str)
	{
		Poco::StringTokenizer t(Poco::replace(str, ",\"", ""), "\"",
				Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);

		double lastQuote, marketCap;
		bool isValid = true;
		if (Poco::NumberParser::tryParseFloat(t[LAST_SALE_INDEX], lastQuote) == false)
		{
			isValid = false;
			//lastQuote = -1;
			//std::cout << t[SYMBOL_INDEX] << " last quote: " << t[LAST_SALE_INDEX] << std::endl;
		}
		if (Poco::NumberParser::tryParseFloat(t[MARKET_CAP_INDEX], marketCap) == false)
		{
			//marketCap = -1;
			//std::cout << t[SYMBOL_INDEX]<< " market capitalization: " << t[LAST_SALE_INDEX] << std::endl;
			isValid = false;
		}

		companiesOfExchange.insert(std::make_pair(t[SYMBOL_INDEX], CompanyData (t[SYMBOL_INDEX], exchange, t[NAME_INDEX],
				lastQuote, marketCap, t[SECTOR_INDEX], t[INDUSTRY_INDEX], t[SUMMARY_QUOTE_INDEX], isValid)));
	} );

	std::cout << exchange << " is done" << std::endl;

	return companiesOfExchange;
}

void CompanyDB::showData()
{
	for (auto it = companies.begin(); it != companies.end(); it++)
	{
		it->second.showCompanyInfo();
	}
}

void CompanyDB::showData(const std::string& ticker)
{
	companies[Poco::toUpper(ticker)].showCompanyInfo();
}

void CompanyDB::populateFinancialData()
{
	for (auto it = companies.begin(); it != companies.end(); it++)
	{
		if (it->second.isValid() == true)
		{
			populateFinancialData(it->first);
		}
	}
}

void CompanyDB::populateFinancialData(const std::string& symbol)
{
	std::string ticker = Poco::toUpper(symbol);
	std::cout << "Financial data for: " << ticker << std::endl;
	try
	{
		CompanyData cd = companies.at(ticker);
		Downloader d("wwwgate0.mot.com", 1080);
		std::vector<std::string> fetchedData;
		fetchedData = d.fetchData(GOOG_FINANCIAL_1 + cd.getExchange() + GOOG_FINANCIAL_2 + ticker + GOOG_FINANCIAL_3);

		std::vector<std::string> temp;

		std::map<std::string, std::map<std::string, float>> incomeStatement;
			FinDataThread t1(ticker, INCOME_STATEMENT, fetchedData, IncomeStatement::CATEGORIES, IncomeStatement::CATEGORIES_PER_SHARE,
				&incomeStatement);
		std::map<std::string, std::map<std::string, float>> balanceSheet;
		FinDataThread t2(ticker, BALANCE_SHEET, fetchedData, BalanceSheet::CATEGORIES, temp,
				&balanceSheet);
		std::map<std::string, std::map<std::string, float>> cashFlow;
		FinDataThread t3(ticker, CASH_FLOW, fetchedData, CashFlow::CATEGORIES, temp,
				&cashFlow);

				Poco::ThreadPool pool;
		pool.start(t1);
		pool.start(t2);
		pool.start(t3);
		pool.joinAll();

		std::cout << ticker << ", is fin_size=" << incomeStatement.size() << std::endl;
		std::cout << ticker << ", bs fin_size=" << balanceSheet.size() << std::endl;
		std::cout << ticker << ", cf fin_size=" << cashFlow.size() << std::endl;

		cd.setIncomeStatement(incomeStatement);
		cd.setBalanceSheet(balanceSheet);
		cd.setCashFlow(cashFlow);
		companies[ticker] = cd;
	}
	catch (std::out_of_range& e)
	{
		std::cout << e.what() << ": Ticker (" << ticker << ") not found" << std::endl;
		return;
	}
	catch (std::exception& e)
	{
		std::cout << "[!!!] Lack of info for ticker " << ticker << std::endl;
		return;
	}
}

int CompanyDB::retrieveFinancialYears(const std::vector<std::string>& fetchedData,
		std::vector<std::string>& years)
{
	const std::string year_str = "12 months ending";
	const std::string multi_str = "except for per share items";
	const std::string million_str = "Millions";
	const std::string thousand_str = "Thousands";

	int counter = 0;
	const int QUANTITY = 4;

	auto it = fetchedData.begin();
	auto end_it = fetchedData.end();

	bool flag = false;
	int multiplier = 1;

	for (; it != end_it; ++it)
	{
//		std::cout << *it << std::endl;
		if (counter < QUANTITY)
		{
			if (Poco::isubstr(*it, year_str) != std::string::npos)
			{
				if (!flag)
				{
					flag = true;
					if (Poco::isubstr(*(it - 3), multi_str) != std::string::npos)
					{
						if (Poco::isubstr(*(it - 3), million_str) != std::string::npos)
						{
							multiplier = 1000000;
						}
						else if (Poco::isubstr(*(it - 3), thousand_str) != std::string::npos)
						{
							multiplier = 1000;
						}
					}
					else
					{
						std::cout << "Error in multiplier" << std::endl;
						throw std::exception();
					}
				}
				Poco::RegularExpression re("[12][0-9][0-9][0-9]-[01]?[0-9]-[0-3]?[0-9]");
				std::string str;
				int n = re.extract(*it, str);
				if (n == 1)
				{
					years.push_back(str);
					counter++;
				}
				else
				{
					std::cout << "Error while parsing year: " << *it << std::endl;
					throw std::exception();
				}
			}
		}
		else
		{
			break;
		}
	}
	return multiplier;
}

void CompanyDB::retrieveFinancials(
		const std::vector<std::string>& fetchedData, const std::vector<std::string>& years,
		const std::vector<std::string>& categories, const std::vector<std::string>& categoriesPerShare, int multiplier, std::map<std::string, std::map<std::string, float>>* financials)
{
	std::string multi_str = "except for per share items";

	auto it = fetchedData.begin();
	auto end_it = fetchedData.end();

	for (; it != end_it; ++it)
	{
		bool found = false;
		Poco::StringTokenizer t(*it, ">", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
		for (auto category : categories)
		{
			if (t.has(category) == true)
			{
				found = true;
				it += 2;
				Poco::RegularExpression re("[-]?[0-9][0-9,\.]*");
				std::string str;
				int n;
				for (auto year : years)
				{
					n = re.extract(*it, str);
					if (n == 1)
					{
						if (str.find("-") != std::string::npos)
						{
							multiplier *= -1;
							Poco::translateInPlace(str, "-", "");
						}
						Poco::translateInPlace(str, ",", "");
						float value = multiplier * Poco::NumberParser::parseFloat(str);
						(*financials) [category] [year] = value;
					}
					else
					{
						Poco::RegularExpression re(">-<");
						n = re.extract(*it, str);
						if (n != 1)
						{
							std::cout << "!!! Error: " << *it << std::endl;
							throw std::exception();
						}
					}
					++it;
				}
				break;
			}
		}
		if (found == true)
		{
			continue;
		}
		for (auto category : categoriesPerShare)
		{
			if (t.has(category) == true)
			{
				it += 2;
				Poco::RegularExpression re("[-]?[0-9][0-9,\.]*");
				std::string str;
				int n;
				int m = 1;
				for (auto year : years)
				{
					n = re.extract(*it, str);
					if (n == 1)
					{
						if (str.find("-") != std::string::npos)
						{
							m *= -1;
							Poco::translateInPlace(str, "-", "");
						}
						Poco::translateInPlace(str, ",", "");
						float value = m * Poco::NumberParser::parseFloat(str);
						(*financials) [category] [year] = value;
					}
					else
					{
						Poco::RegularExpression re(">-<");
						n = re.extract(*it, str);
						if (n != 1)
						{
							std::cout << "!!! Error: " << *it << std::endl;
							throw std::exception();
						}
					}
					++it;
				}
				break;
			}
		}
	}

}


void CompanyDB::populateIncomeStatement(const std::string& ticker)
{
	Downloader d("wwwgate0.mot.com", 1080);
	std::vector<std::string> fetchedData =
				d.fetchData(FIN_STATEMENT_URL_1 + ticker + FIN_STATEMENT_URL_2 + INCOME_STATEMENT_QUERY_URL);

	try
	{
		IncomeStatement is;
//		companies.at(Poco::toUpper(ticker)).setIncomeStatement(is.parseWebPageAndSet(fetchedData));
	}
	catch (std::out_of_range& e)
	{
		std::cout << e.what() << ": Ticker (" << ticker << ") not found" << std::endl;
	}
}

void CompanyDB::populateBalanceSheet(const std::string& ticker)
{
	Downloader d("wwwgate0.mot.com", 1080);
	std::vector<std::string> fetchedData =
				d.fetchData(FIN_STATEMENT_URL_1 + ticker + FIN_STATEMENT_URL_2 + BALANCE_SHEET_QUERY_URL);

	try
	{
		BalanceSheet bs;
//		companies.at(Poco::toUpper(ticker)).setBalanceSheet(bs.parseWebPageAndSet(fetchedData));
	}
	catch (std::out_of_range& e)
	{
		std::cout << e.what() << ": Ticker (" << ticker << ") not found" << std::endl;
	}
}

void CompanyDB::populateCashFlow(const std::string& ticker)
{
	Downloader d("wwwgate0.mot.com", 1080);
	std::vector<std::string> fetchedData =
				d.fetchData(FIN_STATEMENT_URL_1 + ticker + FIN_STATEMENT_URL_2 + CASH_FLOW_QUERY_URL);

	try
	{
		CashFlow cf;
//		companies.at(Poco::toUpper(ticker)).setCashFlow(cf.parseWebPageAndSet(fetchedData));
	}
	catch (std::out_of_range& e)
	{
		std::cout << e.what() << ": Ticker (" << ticker << ") not found" << std::endl;
	}
}

void CompanyDB::populateFinancialRatios(const std::string& ticker)
{
	Downloader d("wwwgate0.mot.com", 1080);
	std::vector<std::string> fetchedData =
		d.fetchData(FIN_STATEMENT_URL_1 + ticker + FIN_STATEMENT_URL_2 + FIN_RATIO_QUERY_URL);

	try
	{
		FinancialRatios fr;
//		companies.at(Poco::toUpper(ticker)).setFinancialRatios(fr.parseWebPageAndSet(fetchedData));
	}
	catch (std::out_of_range& e)
	{
		std::cout << e.what() << ": Ticker (" << ticker << ") not found" << std::endl;
	}
}

void CompanyDB::populateKeyStatistics(const std::string& ticker)
{
	Downloader d("wwwgate0.mot.com", 1080);
	std::vector<std::string> fetchedData =
		d.fetchData(KEY_STATS_URL_1 + ticker + KEY_STATS_URL_2);

	try
	{
		KeyStatistics ks;
		companies.at(Poco::toUpper(ticker)).setKeyStatistics(ks.parseWebPageAndSet(fetchedData));
	}
	catch (std::out_of_range& e)
	{
		std::cout << e.what() << ": Ticker (" << ticker << ") not found" << std::endl;
	}
}

void CompanyDB::saveData() const
{
//	Poco::AutoPtr<Poco::Util::XMLConfiguration> pConf(new Poco::Util::XMLConfiguration("test.xml"));
}
